# -*- coding: utf-8 -*-

import abc
import re

import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery
from readability.readability import Document

import util


class Crawler(object):

    def __init__(self):
        self._logger = None

    def crawl(self, params):
        self._logger = params.get('logger')
        self._logger.info('{%s} start ... %s' % (self.__class__.__name__, util.ustr(params)))

        content = self._crawl(params)

        return content

    def _fetch_content(self, url, params={}, timeout=15):
        content = None

        try:
            res = requests.get(url=url, params=params, timeout=timeout)

            if res.status_code == 200:
                content = res.content
            else:
                self._logger.warn('fetch content failed: code[ %s ]' % res.status_code)

            res.close()
        except Exception as e:
            self._logger.error('fetch content failed: error[ %s ]' % e)

        return content

    def _extract_content(self, data):
        try:
            readable_article = Document(data).summary()

            content = PyQuery(readable_article).text()
            if content:
                text_p = readable_article.replace('<p>', '@@@p@@@').replace('</p>', '+++p+++')
                text_p = re.sub('<.+?>', '', text_p)
                text_p = text_p.replace('@@@p@@@', '<p>').replace('+++p+++', '</p>')
            else:
                self._logger.warn('No content?')
                content = ''
                text_p = '<p></p>'

            return (content, text_p)
        except Exception as e:
            self._logger.error('extract content failed: error[ %s ]' % e)

            return ('', '<p></p>')

    def _fetch_and_extract_content(self, url, params={}):
        return self._extract_content(self._fetch_content(url, params))

    def _structuring_content(self, content, method='lxml'):
        try:
            return BeautifulSoup(content, method)
        except Exception as e:
            self._logger.error('structuring content failed: error[ %s ]' % e)

            return None

    def _replace_content(self, patterns, content):
        for x in patterns:
            s, t = x
            content = re.sub(s, t, content)

        return content

    @abc.abstractmethod
    def _crawl(self, params):
        raise NotImplementedError('call to abstract method %s._crawl' % (self.__class__,))
